#!/usr/bin/env python3

"""
Python3 script to fetch pinyin automatically, based on raw txt files. Results will be saved to dir "result".

Format of the raw txt file should be any of the following three types:
'''
中国
美国 49.9999
德国 97.0 # this is a comment
'''

After running this script, you get the following result (saved as result txt files):
'''
中国 zhong'guo
美国 mei'guo 49.9999
德国 de'guo 97.0
'''

As you can see, the result file can be imported to ibus-libpinyin directly.

To run this script:
1. Put your raw text file with words into "raw_dir".
2. Revise "raw_dir" vairable below.
3. Run this script (in terminal)

Author: Kevin Suo <suokunlong@126.com>
        Jackie Han <jackiehank@163.com>
This script is written specially for libpinyin, but may be used for any other suitable purposes.

Licensed under GNU Public License v3 or above.
"""

import os
from pypinyin import lazy_pinyin, Style

# Please revise this line before run.
raw_dir = r"raw"


def fetch_pinyin(words, sep="'"):
    """Fetch pinyin using pypinyin library
    inputs: words = [['中国'],
                     ['美国', '49.9999'],
                     ['德国', '97.0', '# this is a comment']]
    output: dicts = [['中国', "zhong'guo", None, None],
                     ['美国', "mei'guo", '49.9999', None],
                     ['德国', "de'guo", '97.0', '# this is a comment']]
    """
    print("process words: ", words)

    dicts = []
    for word in words:
        chars = word[0]
        # 使用pypinyin转换拼音，不带声调，用分隔符连接
        pinyin_list = lazy_pinyin(chars, style=Style.NORMAL)
        pinyin = sep.join(pinyin_list)

        l = len(word)
        if l == 1:
            freq = None
            other = None
        elif 1 < l <= 3:
            freq = word[1]
            other = None
            if l == 3:
                other = word[2]
        else:
            raise Exception("lenth of word list > 3: ", word)

        dict = [chars, pinyin, freq, other]
        dicts.append(dict)

    return dicts


def get_words(lines):
    """
    input:  lines = ["中国",
                     "美国 49.9999",
                     "德国 97.0 # this is a comment"]
    return: words = [['中国'],
                     ['美国', '49.9999'],
                     ['德国', '97.0', '# this is a comment']]
    """
    words = []

    for line in lines:
        word = line.split(" ", maxsplit=2)
        chars = word[0]
        if not is_chinese(chars):
            raise Exception(
                "get_words_error", "Non-chinese word chars found in line: ", line
            )

        if len(word) >= 2:
            freq = word[1]
            if not freq.replace(".", "").isdigit():
                raise Exception(
                    "get_words_error", "non-numeric freq chars found in line: ", line
                )

        words.append(word)

    return words


def is_chinese(uchar):
    """判断一个unicode是否是汉字
    http://blog.csdn.net/qinbaby/article/details/23201883
    """
    if uchar >= "\u4e00" and uchar <= "\u9fa5":
        return True
    else:
        return False


def check_line(line):
    """ """
    line = line.strip()
    if line.startswith("#") or line == "":
        pass
    else:
        word = line.split(" ", maxsplit=2)
        chars = word[0]

        if not is_chinese(chars):
            print("Line error: ", "Non-chinese word chars found in line: ", line)
            return False

        if len(word) >= 2:
            freq = word[1]
            if not freq.replace(".", "").isdigit():
                print("get_words_error", "non-numeric freq chars found in line: ", line)
                return False

    return True


def check_raw_file(raw_file):
    """ """
    with open(raw_file, "r") as f_in:
        lines = f_in.readlines()
        errors = []
        for i in range(len(lines)):
            if check_line(lines[i]):
                pass
            else:
                errors.append([i + 1, lines[i]])

    if len(errors) > 0:
        print("check raw file erros are: ", errors, "\n", raw_file)
        return False
    else:
        return True


def check_raw_files(raw_dir):
    """ """
    for file_name in os.listdir(raw_dir):
        # print("checking raw file: ", file_name)
        raw_file = os.path.join(raw_dir, file_name)
        if os.path.isdir(raw_file):
            pass
        else:
            if check_raw_file(raw_file) == False:
                raise Exception("Raw File Check Failed: ", raw_file)


def process_raw_file(raw_file, result_file):
    """1. 一行一行读取
    2. 碰到comment行或空行，如果lines_current里有要索取的词，则执行索取操作，写入索取结果，清零lines_current, 然后再写入comment或空行。
    3. 每max_lines_per_fetch行时，强制索取。
    4. 到结尾时，索取剩余的lines_current中的词。
    """
    if os.path.isdir(raw_file) or os.path.isdir(result_file):
        return False

    with open(result_file, "w") as f_out:
        with open(raw_file, "r") as f_in:
            lines = f_in.readlines()
            max_lines_per_fetch = 200
            i = 1
            lines_current = []

            for line in lines:
                line = line.strip()
                if line.startswith("#") or line == "":
                    if len(lines_current) == 0:  # there is no words to fetch
                        f_out.write(line + "\n")
                    else:
                        words = get_words(lines_current)
                        dicts = fetch_pinyin(words)
                        write_dicts(f_out, dicts)
                        lines_current.clear()
                        # remember to write the comment or blank line after fetch
                        f_out.write(line + "\n")
                else:
                    lines_current.append(line)
                    if len(lines_current) >= max_lines_per_fetch or i == len(lines):
                        words = get_words(lines_current)
                        dicts = fetch_pinyin(words)
                        write_dicts(f_out, dicts)
                        lines_current.clear()
                i += 1


def write_dicts(f_out, dicts):
    """ """
    for dict in dicts:
        chars = dict[0]
        pinyin = dict[1]
        freq = dict[2]
        # "other" is disregarded in below process.
        other = dict[3]
        if freq == None:
            f_out.write(dict[0] + " " + dict[1] + "\n")
        else:
            f_out.write(chars + " " + pinyin + " " + freq + "\n")


def process_raw_files(raw_dir):
    """ """
    # check for errors in raw files before do anything else
    check_raw_files(raw_dir)

    result_dir = os.path.join(raw_dir, "result")
    if not os.path.isdir(result_dir):
        os.mkdir(result_dir)

    for file_name in os.listdir(raw_dir):
        print("Catch: ", file_name)
        if os.path.isdir(file_name):
            pass
        else:
            raw_file = os.path.join(raw_dir, file_name)
            result_file = os.path.join(raw_dir, "result", file_name)
            process_raw_file(raw_file, result_file)


def main():
    process_raw_files(raw_dir)


if __name__ == "__main__":
    main()
